# coding = utf8
from numpy import *
import sys
import os
from sklearn.datasets.base import  Bunch
import pickle as pickle
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from imp import reload
reload(sys)

'''
读取和写入bunch对象
'''
#读取和写入Bunch对象的函数
def readbunchobj(path):
    file_obj = open(path, "rb")
    bunch = pickle.load(file_obj)
    file_obj.close()
    return bunch

#写入bunch对象
def writebunchobj(path, bunchobj):
    file_obj = open(path, "wb")
    pickle.dump(bunchobj, file_obj)
    file_obj.close()

'''
从训练集生成TF-IDF向量词袋
'''
#导入分词后的词向量bunch对象
path = ""
bunch = readbunchobj(path)
#构建tf-idf词向量空间对象
tfidfspace = Bunch(target_name=bunch.targent_name, label=bunch.label, filenames=bunch.filenames, tdm=[], vocabulary=[])
#使用tfidfvectorizer初始化向量空间模型
vectorizer = TfidfVectorizer(stop_words=stpwrdlst, sublinear_tf=True, max_df=0.5)
transformer = TfidfTransformer()
#文本转化为词频矩阵，单独保存字典文件
tfidfspace.tdm = vectorizer.fit_transform(bunch.contents)
tfidfspace.vocabulary = vectorizer.vocabulary_


space_path = ""
writebunchobj(space_path, tfidfspace)

